package com.deepbreath.spider.pm25in;

import java.sql.Timestamp;
import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.lang3.StringEscapeUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.deepbreath.entity.CitySite;
import com.deepbreath.spider.AbstractProcessor;

import com.deepbreath.common.Ctype;
import com.deepbreath.spider.HanyuPinyinHelper;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class CitySiteProcessor extends AbstractProcessor<Void> implements PageProcessor {
	// 用来处理不能抽取到的城市
	public final static String TIANQI_URL = "http://tianqi.2345.com/t/searchCity.php?q=%s&pType=local";
	public final static String CITY_KEY = "city";
	public final static String PROVINCE_KEY = "province";

	public final static String NET_URL = "http://www.pm25.in/";
	public final static String RANK_URL = "rank";

	private Site site = null;
	private Spider spider = null;

	private Map<String, CitySite> DATA_MAP = new ConcurrentHashMap<String, CitySite>();

	public Collection<CitySite> getDATA_MAP() {
		return DATA_MAP.values();
	}

	@Override
	public void process(Page page) {
		try {
			Document doc = page.getHtml().getDocument();
			String url = page.getUrl().toString();
			if (url.endsWith(RANK_URL)) {// 分析http://www.pm25.in/rank页面
				System.out.println("正在搜索城市所属省份信息,请等待……");
				Elements elements = doc.select("table.table>tbody>tr>td>a");
				for (Element element : elements) {
					String city = element.text();
					String id = element.attr("href").substring(1);
					Timestamp time = new Timestamp(System.currentTimeMillis());

					Request request = new Request(NET_URL + id);
					request.addHeader(CITY_KEY, city);

					// 解决特殊情况没办法搜到对应的省
					String keywords = city;
					keywords = keywords.endsWith("地区") ? keywords.substring(0, keywords.length() - 2) : keywords;
					keywords = keywords.endsWith("藏族自治州") ? keywords.substring(0, keywords.length() - 5) : keywords;
					keywords = keywords.endsWith("州") ? keywords.substring(0, keywords.length() - 1) : keywords;

					String result = Jsoup.connect(String.format(TIANQI_URL, keywords)).get().text();
					JSONArray jsonArray = JSON.parseObject(result).getJSONArray("res");
					String text = null;
					if (jsonArray.size() > 0) {
						text = StringEscapeUtils.unescapeJava(jsonArray.getJSONObject(0).getString("text"));
						text = text.substring(text.lastIndexOf("-") + 1, text.lastIndexOf("<")).replace(" ", "");
						request.addHeader(PROVINCE_KEY, text);
					}

					CitySite cityEntity = new CitySite(id, Ctype.CITY.toString(), text, city, "-", time);
					DATA_MAP.put(id, cityEntity);

					// 继续抓取当前城市的具体站点信息
					page.addTargetRequest(request);
					Thread.sleep(60);// 防止被查询省份的服务发现
				}
				System.out.println("搜索城市所属省份信息结束!");
			} else {// 分析某个城市的检查站点页面
				String city = page.getRequest().getHeaders().get(CITY_KEY);
				String province = page.getRequest().getHeaders().get(PROVINCE_KEY);
				Elements elements = doc.select("table#detail-data>tbody>tr");
				for (Element element : elements) {
					String position = element.select("td").first().text();
					Timestamp time = new Timestamp(System.currentTimeMillis());
					String id = HanyuPinyinHelper.toHanyuPinyin(position);
					DATA_MAP.put(id, new CitySite(id, Ctype.POSITION.toString(), province, city, position, time));
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	@Override
	public Site getSite() {
		return site;
	}

	@Override
	public void initialize(Void args) {
		site = Site.me().setSleepTime(1000).setTimeOut(3000).setRetryTimes(5);
		spider = Spider.create(this).addUrl(NET_URL + RANK_URL).thread(5);
	}

	@Override
	public void start() {
		spider.run();
	}

	@Override
	public void shutdown() {
		spider.close();
	}

	public static void main(String[] args) throws Exception {
		CitySiteProcessor processor = new CitySiteProcessor();
		processor.initialize(null);
		processor.start();
		Thread.sleep(2000);
		System.out.println(processor.getDATA_MAP());
		System.out.println(JSON.toJSONString(processor.getDATA_MAP()));
	}
}
